<?
require_once('unicode.php');

function print_entry($key,$entry)
{
  $line = sprintf("'%s': [",$key);
  for($i=0;$i<count($entry);$i++) {
    $value = $entry[$i];
    if($value >= 0xd800 && $value <= 0xdfff) die("Error\n");
    if($value > 0x10ffff) die("Error\n");
    $line .= sprintf("0x%x,",$value);
  }
  $line = rtrim($line,',');
  $line .= "],";
  printf("%s\n",$line);
}
function load_unicode_pop($file)
{
  $fp = fopen($file,'r');

  $p = 0;
  $unicode_pop = array();
  while($data = fscanf($fp,"0x%x\n")) {
    $unicode_pop[$data[0]] = $p++;
  }
  fclose($fp);
  return $unicode_pop;
}

if(count($argv) < 2) return;

$unicode_pop = load_unicode_pop('big5pop.txt');

$fp = fopen($argv[1],'r');

$entries = array();
while($line = fgets($fp)) {
  $line = rtrim($line,"\n");
  $pos = strpos($line,' ');
  $txt = substr($line,0,$pos);
  $data = substr($line,$pos+1);
  $scalar = readutf8($data);

  if(key_exists($txt,$entries)) {
    $entry = $entries[$txt];
    $entry[] = $scalar;
    $entries[$txt] = $entry;
  }
  else {
    $entry = array();
    $entry[] = $scalar;
    $entries[$txt] = $entry;
  }
}
fclose($fp);

printf("table: {\n");
// print first 2000 words
foreach($entries as $key => $entry) {
  $unicode = $entry[0];
  if($unicode_pop[$unicode] <= 2000) print_entry($key,$entry);
  else print_entry($key,array());
}
printf("},\n");

// print 2001-5000 words
printf("//BEGIN_COMPONENT{table0}\n");
printf("table0: {\n");
foreach($entries as $key => $entry) {
  $unicode = $entry[0];
  if(2001 <= $unicode_pop[$unicode] && $unicode_pop[$unicode] <= 5000) print_entry($key,$entry);
}
printf("},\n");
printf("//END_COMPONENT\n");

// print 5001-9000 words
printf("//BEGIN_COMPONENT{table1}\n");
printf("table1: {\n");
foreach($entries as $key => $entry) {
  $unicode = $entry[0];
  if(5001 <= $unicode_pop[$unicode] && $unicode_pop[$unicode] <= 9000) print_entry($key,$entry);
}
printf("},\n");
printf("//END_COMPONENT\n");

// print remaining
printf("//BEGIN_COMPONENT{table2}\n");
printf("table2: {\n");
foreach($entries as $key => $entry) {
  $unicode = $entry[0];
  if(9001 <= $unicode_pop[$unicode]) print_entry($key,$entry);
}
printf("},\n");
printf("//END_COMPONENT\n");

?>
